package dmp.cjh.url.classfy;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.Logger;

public class UrlClassfy {
	public static class UrlFilterMap extends Mapper<LongWritable, Text, Text, Text> {

		public static Logger logger2 = Logger.getLogger(Mapper.class);
		@Override
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			String line = value.toString();
			String[] words = line.split("\\|");
			String url = words[27];
			
			logger2.info("-------------------------"+url);
			
			
			//汽车之家搜索url
			if (url.contains("http://app.api.autohome.com.cn/autov5.0.5/sou/search.ashx")) {
				context.write(new Text("qczj_search_url"), new Text(line));
			}
			//汽车之家按品牌url
			if (url.contains("http://app.api.autohome.com.cn/autov5.1.0/cars/seriessummary")) {
				context.write(new Text("qczj_search_by_brand"), new Text(line));
			}
			//汽车之家询价url
			if (url.contains("http://app.api.autohome.com.cn/autov5.1.0/cars/specslist") || url.contains("http://app.api.autohome.com.cn/autov5.1.0/dealer/AskDealersPrice.ashx")) {
				context.write(new Text("qczj_search_consult_price"), new Text(line));
			}
			//汽车报价大全按品牌搜索url
			if (url.contains("http://api.app.yiche.com/webapi/QueryParam.ashx?serialid") || url.contains("http://api.app.yiche.com/webapi/api.ashx?method=bit.seriessummary")) {
				context.write(new Text("qcbjdq_search_by_brand"), new Text(line));
			}
			//汽车报价大全询底价url
			if (url.contains("http://dealer.app.yiche.com/api.ashx?method=bit.dealer.infolist&carid")) {
				context.write(new Text("qcbjdq_search_lowest_price"), new Text(line));
			}
			//惠买车询底价url
			if (url.contains("http://m.ycapp.huimaiche.com/order") || url.contains("http://m.ycapp.huimaiche.com/concepts/")) {
				context.write(new Text("hmc_search_lowest_price"), new Text(line));
			}
			//惠买车app中询底价url
			if (url.contains("http://api.huimaiche.com/User/Util/GetDealerListAndColorInfoByCarId/") ) {
				context.write(new Text("hmc_app_search_lowest_price"), new Text(line));
			}
		}
	}

	
	public static void main(String[] args) throws Exception {
		Configuration conf=new Configuration();
		conf.set("mapreduce.job.queuename", "ven2"); //设置队列
	
		
	
		Job job = Job.getInstance(conf);
		
		job.setOutputFormatClass(MyOutputFormat.class);//自定义outputFormat
		
		job.setJobName("urlfilter");
		job.setJarByClass(UrlClassfy.class);
		
		job.setMapperClass(UrlFilterMap.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);
		
		job.setOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);
		
		
		SequenceFileInputFormat.setInputPaths(job, new Path(args[0]));
		job.setInputFormatClass(SequenceFileInputFormat.class);
		
		Path out=new Path(args[1]);
		FileOutputFormat.setOutputPath(job, out);
		
		FileSystem fs=FileSystem.get(conf);
		if(fs.exists(out)){
			fs.delete(out, true);
		}
		
		job.waitForCompletion(true);
	}
	
}
